{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HTML解析入门及准备URL生成连续技\n",
    "\n",
    "*  本周主要内容：批量抓取页面基础及技巧\n",
    "*  上周主要内容：HTML解析（parse HTML）及准备URL生成连续技\n",
    "*  20春_Web数据挖掘_week04\n",
    "*  电子讲义设计者：廖汉腾, 许智超\n",
    "<br/>\n",
    "<br/>\n",
    "\n",
    "-----\n",
    "## 复习\n",
    "\n",
    "复习：上周内容，实践\n",
    "\n",
    "* 翻页：参数字典的拆解\n",
    "* 翻页：系统性迭代\n",
    "* 翻页：数据备份与整合\n",
    "\n",
    "-----\n",
    "## 本周内容及学习目标\n",
    "\n",
    "搜狗微信：搜索关键字获取微信公众号相关文章\n",
    "  \n",
    "### 目标\n",
    "1. 使用 requests-html 爬取并存取网页文字档，查找[requests-html 中文文档](https://cncert.github.io/requests-html-doc-cn/#/)\n",
    "2. 熟悉 [xpath 语法](https://www.w3cschool.cn/xpath/xpath-syntax.html)丶[xpath 节点](https://www.w3cschool.cn/xpath/xpath-nodes.html)\n",
    "3. 使用 [xpath cheatsheet](https://devhints.io/xpath)\n",
    "  * 在 Chrome Inspector 使用\n",
    "  * 在 requests-html (Python) 使用\n",
    "4. 简易使用 [pd.DataFrame](https://www.pypandas.cn/doc/getting_started/dsintro.html#dataframe)\n",
    "5. 参数字典的拆解与迭代\n",
    "6. 翻页数据备份与整合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>\n",
       "/* 本电子讲义使用之CSS */\n",
       "div.code_cell {\n",
       "    background-color: #e5f1fe;\n",
       "}\n",
       "div.cell.selected {\n",
       "    background-color: #effee2;\n",
       "    font-size: 2rem;\n",
       "    line-height: 2.4rem;\n",
       "}\n",
       "div.cell.selected .rendered_html table {\n",
       "    font-size: 2rem !important;\n",
       "    line-height: 2.4rem !important;\n",
       "}\n",
       ".rendered_html pre code {\n",
       "    background-color: #C4E4ff;   \n",
       "    padding: 2px 25px;\n",
       "}\n",
       ".rendered_html pre {\n",
       "    background-color: #99c9ff;\n",
       "}\n",
       "div.code_cell .CodeMirror {\n",
       "    font-size: 2rem !important;\n",
       "    line-height: 2.4rem !important;\n",
       "}\n",
       ".rendered_html img, .rendered_html svg {\n",
       "    max-width: 60%;\n",
       "    height: auto;\n",
       "    float: right;\n",
       "}\n",
       "\n",
       ".rendered_html img[src*=\"#full\"], .rendered_html svg[src*=\"#full\"] {\n",
       "    max-width: 100%;\n",
       "    height: auto;\n",
       "    float: none;\n",
       "}\n",
       "\n",
       ".rendered_html img[src*=\"#thumbnail\"], .rendered_html svg[src*=\"#thumbnail\"] {\n",
       "    max-width: 15%;\n",
       "    height: auto;\n",
       "}\n",
       "\n",
       "/* Gradient transparent - color - transparent */\n",
       "hr {\n",
       "    border: 0;\n",
       "    border-bottom: 1px dashed #ccc;\n",
       "}\n",
       ".emoticon{\n",
       "    font-size: 5rem;\n",
       "    line-height: 4.4rem;\n",
       "    text-align: center;\n",
       "    vertical-align: middle;\n",
       "}\n",
       ".bg-split_apply_comine {\n",
       "    width: 500px;     \n",
       "    height: 300px;\n",
       "    background: url('02_split-apply-comine_500x300.png') -10px -10px;\n",
       "    float: right;\n",
       "}\n",
       ".bg-comine {\n",
       "    width: 175px;\n",
       "    height: 150px;\n",
       "    background: url('02_split-apply-comine_500x300.png') -280px -80px;\n",
       "    float: right;\n",
       "}\n",
       ".bg-apply {\n",
       "    width: 155px;\n",
       "    height: 225px;\n",
       "    background: url('02_split-apply-comine_500x300.png') -160px -30px;\n",
       "    float: right;\n",
       "}\n",
       ".bg-split {\n",
       "    width: 205px;\n",
       "    height: 225px;\n",
       "    background: url('02_split-apply-comine_500x300.png') -10px -30px;\n",
       "    float: right;\n",
       "}\n",
       ".break {\n",
       "                   page-break-after: right; \n",
       "                   width:700px;\n",
       "                   clear:both;\n",
       "}\n",
       "</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%html\n",
    "<style>\n",
    "/* 本电子讲义使用之CSS */\n",
    "div.code_cell {\n",
    "    background-color: #e5f1fe;\n",
    "}\n",
    "div.cell.selected {\n",
    "    background-color: #effee2;\n",
    "    font-size: 2rem;\n",
    "    line-height: 2.4rem;\n",
    "}\n",
    "div.cell.selected .rendered_html table {\n",
    "    font-size: 2rem !important;\n",
    "    line-height: 2.4rem !important;\n",
    "}\n",
    ".rendered_html pre code {\n",
    "    background-color: #C4E4ff;   \n",
    "    padding: 2px 25px;\n",
    "}\n",
    ".rendered_html pre {\n",
    "    background-color: #99c9ff;\n",
    "}\n",
    "div.code_cell .CodeMirror {\n",
    "    font-size: 2rem !important;\n",
    "    line-height: 2.4rem !important;\n",
    "}\n",
    ".rendered_html img, .rendered_html svg {\n",
    "    max-width: 60%;\n",
    "    height: auto;\n",
    "    float: right;\n",
    "}\n",
    "\n",
    ".rendered_html img[src*=\"#full\"], .rendered_html svg[src*=\"#full\"] {\n",
    "    max-width: 100%;\n",
    "    height: auto;\n",
    "    float: none;\n",
    "}\n",
    "\n",
    ".rendered_html img[src*=\"#thumbnail\"], .rendered_html svg[src*=\"#thumbnail\"] {\n",
    "    max-width: 15%;\n",
    "    height: auto;\n",
    "}\n",
    "\n",
    "/* Gradient transparent - color - transparent */\n",
    "hr {\n",
    "    border: 0;\n",
    "    border-bottom: 1px dashed #ccc;\n",
    "}\n",
    ".emoticon{\n",
    "    font-size: 5rem;\n",
    "    line-height: 4.4rem;\n",
    "    text-align: center;\n",
    "    vertical-align: middle;\n",
    "}\n",
    ".bg-split_apply_comine {\n",
    "    width: 500px;     \n",
    "    height: 300px;\n",
    "    background: url('02_split-apply-comine_500x300.png') -10px -10px;\n",
    "    float: right;\n",
    "}\n",
    ".bg-comine {\n",
    "    width: 175px;\n",
    "    height: 150px;\n",
    "    background: url('02_split-apply-comine_500x300.png') -280px -80px;\n",
    "    float: right;\n",
    "}\n",
    ".bg-apply {\n",
    "    width: 155px;\n",
    "    height: 225px;\n",
    "    background: url('02_split-apply-comine_500x300.png') -160px -30px;\n",
    "    float: right;\n",
    "}\n",
    ".bg-split {\n",
    "    width: 205px;\n",
    "    height: 225px;\n",
    "    background: url('02_split-apply-comine_500x300.png') -10px -30px;\n",
    "    float: right;\n",
    "}\n",
    ".break {\n",
    "                   page-break-after: right; \n",
    "                   width:700px;\n",
    "                   clear:both;\n",
    "}\n",
    "</style>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 基本模块\n",
    "import pandas as pd\n",
    "from requests_html import HTMLSession\n",
    "from pandas.api.types import is_numeric_dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 缺模块，补充"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "#缺模块，补充\n",
    "from urllib.parse import urlparse, parse_qs \n",
    "import time\n",
    "from random import random"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 搜狗微信：文章搜索(模仿liepin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>UX、UI、产品设计、平面设计、交互设计,它们到底是什么关系?</td>\n",
       "      <td>互联网的兴起使交互设计成为了热门专业,相信很多同学最初接触这个专业时都有很多困惑为什么交互设...</td>\n",
       "      <td>BEAUXARTS</td>\n",
       "      <td>1586314278</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-08 02:51:18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>谷歌如何筛选UX设计师——谷歌UX主管访谈</td>\n",
       "      <td>南山可本次译文是关于交互设计基金会对谷歌设计主管的一次访谈,虽然标题是关于UX设计师,但事实...</td>\n",
       "      <td>学UI网</td>\n",
       "      <td>1582463191</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-02-23 13:06:31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>雷克萨斯UX,细节之处见真美</td>\n",
       "      <td></td>\n",
       "      <td>浙江广通雷克萨斯</td>\n",
       "      <td>1585534290</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-03-30 02:11:30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>全新UX丨洗练美学,雕刻时光</td>\n",
       "      <td>更多优惠政策尽在广州永佳雷克萨斯详情请咨询:020-39231888</td>\n",
       "      <td>永佳雷克萨斯</td>\n",
       "      <td>1586080818</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-05 10:00:18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>雷克萨斯UX,细节之处见真美</td>\n",
       "      <td>雷克萨斯UX,细节之处见真美</td>\n",
       "      <td>龙岩美东雷克萨斯</td>\n",
       "      <td>1586339193</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-08 09:46:33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>雷克萨斯UX,细节之处见真美</td>\n",
       "      <td>领未见,探非凡.宜兴伟杰雷克萨斯以至诚之心呈现充满激情与质感的生活方式,和您一同感知和享受触...</td>\n",
       "      <td>宜兴伟杰雷克萨斯</td>\n",
       "      <td>1585913592</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-03 11:33:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>全新UX丨感知细节处的温度与美感</td>\n",
       "      <td></td>\n",
       "      <td>永佳雷克萨斯</td>\n",
       "      <td>1586685626</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-12 10:00:26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>雷克萨斯UX | 大繁至简,拙朴美学</td>\n",
       "      <td>LEXUS·UX</td>\n",
       "      <td>上海美跃雷克萨斯特许经销店</td>\n",
       "      <td>1586252010</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-07 09:33:30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AxureUX三周年红包雨现金抽奖活动</td>\n",
       "      <td>专业的交互原型素材原创分享平台www.AxureUX.com</td>\n",
       "      <td>AxureUX</td>\n",
       "      <td>1583378526</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-03-05 03:22:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>雷克萨斯UX,细节之处见真美</td>\n",
       "      <td>永佳-专注汽车21年技术精湛 值得信赖买雷克萨斯,维修/保养雷克萨斯就选东莞永佳1</td>\n",
       "      <td>东莞永佳雷克萨斯</td>\n",
       "      <td>1586244780</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-07 07:33:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                标题  \\\n",
       "0  UX、UI、产品设计、平面设计、交互设计,它们到底是什么关系?   \n",
       "1            谷歌如何筛选UX设计师——谷歌UX主管访谈   \n",
       "2                   雷克萨斯UX,细节之处见真美   \n",
       "3                   全新UX丨洗练美学,雕刻时光   \n",
       "4                   雷克萨斯UX,细节之处见真美   \n",
       "5                   雷克萨斯UX,细节之处见真美   \n",
       "6                 全新UX丨感知细节处的温度与美感   \n",
       "7               雷克萨斯UX | 大繁至简,拙朴美学   \n",
       "8              AxureUX三周年红包雨现金抽奖活动   \n",
       "9                   雷克萨斯UX,细节之处见真美   \n",
       "\n",
       "                                                  摘要         source  \\\n",
       "0  互联网的兴起使交互设计成为了热门专业,相信很多同学最初接触这个专业时都有很多困惑为什么交互设...      BEAUXARTS   \n",
       "1  南山可本次译文是关于交互设计基金会对谷歌设计主管的一次访谈,虽然标题是关于UX设计师,但事实...           学UI网   \n",
       "2                                                          浙江广通雷克萨斯   \n",
       "3                 更多优惠政策尽在广州永佳雷克萨斯详情请咨询:020-39231888         永佳雷克萨斯   \n",
       "4                                     雷克萨斯UX,细节之处见真美       龙岩美东雷克萨斯   \n",
       "5  领未见,探非凡.宜兴伟杰雷克萨斯以至诚之心呈现充满激情与质感的生活方式,和您一同感知和享受触...       宜兴伟杰雷克萨斯   \n",
       "6                                                            永佳雷克萨斯   \n",
       "7                                           LEXUS·UX  上海美跃雷克萨斯特许经销店   \n",
       "8                     专业的交互原型素材原创分享平台www.AxureUX.com        AxureUX   \n",
       "9          永佳-专注汽车21年技术精湛 值得信赖买雷克萨斯,维修/保养雷克萨斯就选东莞永佳1       东莞永佳雷克萨斯   \n",
       "\n",
       "           时间                                              公众号链接  \\\n",
       "0  1586314278  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "1  1582463191  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "2  1585534290  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "3  1586080818  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "4  1586339193  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "5  1585913592  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "6  1586685626  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "7  1586252010  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "8  1583378526  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "9  1586244780  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "\n",
       "                                                文章链接                格式化时间  \n",
       "0  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-08 02:51:18  \n",
       "1  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-02-23 13:06:31  \n",
       "2  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-03-30 02:11:30  \n",
       "3  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-05 10:00:18  \n",
       "4  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-08 09:46:33  \n",
       "5  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-03 11:33:12  \n",
       "6  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-12 10:00:26  \n",
       "7  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-07 09:33:30  \n",
       "8  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-03-05 03:22:06  \n",
       "9  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-07 07:33:00  "
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A-1 query 为关键字选取 以python为例\n",
    "\n",
    "url = \"https://weixin.sogou.com/weixin\"\n",
    "session = HTMLSession()\n",
    "payload = {\n",
    "    \"query\":\"UX\",\n",
    "    \"type\":\"2\", # 2 文章、1 公众号\n",
    "    \"page\":10  \n",
    "}\n",
    "\n",
    "r = session.get( url, params = payload, )\n",
    "\n",
    "\n",
    "主要元素 =  r.html.xpath( '//ul[@class=\"news-list\"]/li')\n",
    "\n",
    "dict_xpaths = {\n",
    "    'text':{\n",
    "        'source':'//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/a/text()',\n",
    "        '时间':'//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/@t',       \n",
    "    },\n",
    "    'text_content':{\n",
    "        '标题':'//div[contains(@class,\"txt-box\")]/h3/a',\n",
    "        '摘要':'//div[contains(@class,\"txt-box\")]/p'\n",
    "    },\n",
    "    'href':{\n",
    "        \"公众号链接\":'//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/a/@href',\n",
    "        \"文章链接\":'//div[contains(@class,\"txt-box\")]/h3/a/@href',\n",
    "    }\n",
    "}\n",
    "\n",
    "\n",
    "def get_e_text_content(_xpath_):\n",
    "    # 高级列表推导\n",
    "    暂存结果 = [e.xpath(_xpath_)[0].lxml.text_content() for e in 主要元素]\n",
    "\n",
    "    return(暂存结果)\n",
    "\n",
    "def get_e_text(_xpath_):\n",
    "    暂存结果 = [\"\".join([x.strip() if type(x) is str else x.text.strip() for x in e.xpath(_xpath_)]) for e in 主要元素]\n",
    "    return(暂存结果)\n",
    "\n",
    "def get_e_href(_xpath_):\n",
    "    # 高级列表推导\n",
    "    暂存结果 = [\"https://weixin.sogou.com\"+item\\\n",
    "            for e in 主要元素\\\n",
    "            for item in e.xpath(_xpath_)]  #原代码打开运行后报错，修改了for循环的顺序\n",
    "    return(暂存结果)\n",
    "\n",
    "数据字典 = dict()\n",
    "# content\n",
    "数据字典 = {k:get_e_text_content(v) for k,v in dict_xpaths['text_content'].items()}\n",
    "# text\n",
    "数据字典.update({k:get_e_text(v) for k,v in dict_xpaths['text'].items()})\n",
    "# href\n",
    "数据字典.update({k:get_e_href(v) for k,v in dict_xpaths['href'].items()})\n",
    "\n",
    "数据 = pd.DataFrame(数据字典)\n",
    "数据 = 数据.assign(\n",
    "    格式化时间 = lambda x: pd.to_datetime(x[\"时间\"], unit='s').dt.strftime('%Y-%m-%d %H:%M:%S')\n",
    ")\n",
    "数据\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open (\"weixin_UX.html\", encoding = \"utf8\", mode = \"w\") as fp:\n",
    "    fp.write(r.html.html)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## xpath的具体操作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['BEAUXARTS',\n",
       " '学UI网',\n",
       " '浙江广通雷克萨斯',\n",
       " '永佳雷克萨斯',\n",
       " '龙岩美东雷克萨斯',\n",
       " '宜兴伟杰雷克萨斯',\n",
       " '永佳雷克萨斯',\n",
       " '上海美跃雷克萨斯特许经销店',\n",
       " 'AxureUX',\n",
       " '东莞永佳雷克萨斯']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 来源的xpath解析结果\n",
    "r.html.xpath('//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/a/text()')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['、UI、产品设计、平面设计、交互设计,它们到底是什么关系?',\n",
       " '谷歌如何筛选',\n",
       " '设计师——谷歌',\n",
       " '主管访谈',\n",
       " '雷克萨斯',\n",
       " ',细节之处见真美',\n",
       " '全新',\n",
       " '丨洗练美学,雕刻时光',\n",
       " '雷克萨斯',\n",
       " ',细节之处见真美',\n",
       " '雷克萨斯',\n",
       " ',细节之处见真美',\n",
       " '全新',\n",
       " '丨感知细节处的温度与美感',\n",
       " '雷克萨斯',\n",
       " ' | 大繁至简,拙朴美学',\n",
       " 'AxureUX三周年红包雨现金抽奖活动',\n",
       " '雷克萨斯',\n",
       " ',细节之处见真美']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 标题？1、em text()? + a/text()?\n",
    "# 某一条<li>信息,再进行上一步 if else\n",
    "r.html.xpath('//div[contains(@class,\"txt-box\")]/h3/a/text()')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'UX、UI、产品设计、平面设计、交互设计,它们到底是什么关系?'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.html.xpath('//div[contains(@class,\"txt-box\")]/h3/a')[0].lxml.text_content()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'甚至UX在英国起始价29,905英镑贵上不少,只能说艺术是无价的!'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.html.xpath('//div[contains(@class,\"txt-box\")]/p')[0].lxml.text_content()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdzvlH_a5r65klUuXc2en3Gq_QQkGxcvz01GPlrCXgVybgzbjVvui9cEmt8NBnZRMT3pMkZ85Vs_1kiRUUmb8I5eHjvCaaaY329gr5NzriOvpxZbBnrZzPuJKUiiWOv1GPM&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdzyTcpXFbALUMDXrxlDu4ACaZ-0tHoZsNqxM98YuihUyW1c7TPu0OBx791bLKE-JHMVH-zu5ukyLKYPTNkaddccKymIkEO6BIyHV2HHO0rsZYPy9sdpM6LE6UiiWOv1GPM&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdzLu6p_YHPvqMomX8Jw7s4P98oVSgz3ujg8lI21epsCrqpdk52BWXqQqLvgu3IyWSg5ICIBmXgUSYtTKEfIeb4UiV3jY6jd7Nu1gAxELCSyrMOa501LnEsJWRTpLLTNgHY&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdziNSiL1Avg2ao5rRQX6Hn9SdOk03IphjLaq2-I0yIEo0BfW3Gl-Ne4SHmLj5mnxePnYjhdLG3If-OCgmU-0ZzS1_rPAbBl22Zf_WUciGnMWmUr-ms_JrBqm4OZCnxixtX&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdz2wB3jvhVko_T8ohFL1tGOQh9YzFn6dUMO96SkGcuP9o8DGuuar5-2HRqwvn2cWiqlktbSdI8BGrQDh88nrnM3aVe14Nvsc6lBBg0WAPLmOtmD0-kp0bjg-O00efWrWmm&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdzXMqhETrsdE_RtAPw38mJvPrk99_X7i32HVbaTBmFvVXWokMAVF8SQl22rqYF0afKHpcUOJLPNyGnQg_KWVJ0E6Hqr38LfhMGDdKHKMYwauXSFL9FKDKXlOO00efWrWmm&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdziNSiL1Avg2ao5rRQX6Hn9SdOk03IphjLaq2-I0yIEo0BfW3Gl-Ne4SHmLj5mnxePnYjhdLG3If-OCgmU-0ZzS1_rPAbBl22Zf_WUciGnMWmUr-ms_JrBqm4OZCnxixtX&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdzd8w_aBGyrdDMdhZ06333pbkkKpO8LNnbiEJwA_cW5xILqTSwey0_vK_u_elOboRfIvjX-DV_CXrb6bFIfvVI73pF-ffdx6BuEIZquLMprQDX9HJnGkd9c-O00efWrWmm&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdzZP0DOtsf9_d3n_V_UvYpiGelxaBOYqA2fkAH3nxwaAjPMjpgqhGyWYD1uY3rztj-lydD-1o0_fw9-Tl0CLbolzqiYUQshEIZJz_5X0eKzIajL_AdcVHUlKUiiWOv1GPM&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdziPV4qVS8Ms4dIT8Q-BYpGIbIwSHWLyo54r_EZH0SgAoVx9k_A4MqqL4XPJLw62WrU4s0huyy2TPiE4_Rpa5vk9HIQxCEiZeJgHgpPruSNivYa_HL52XI_eO00efWrWmm&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D']"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.html.xpath('//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/a/@href') #链接（是一个list）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://weixin.sogou.com/weixin//link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdzvlH_a5r65klUuXc2en3Gq_QQkGxcvz01GPlrCXgVybgzbjVvui9cEmt8NBnZRMT3pMkZ85Vs_1kiRUUmb8I5eHjvCaaaY329gr5NzriOvpxZbBnrZzPuJKUiiWOv1GPM&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 公众号链接\n",
    "'https://weixin.sogou.com/weixin/'+ r.html.xpath('//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/a/@href')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 公众号文章链接"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 时间"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 图片"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'}"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.request.headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A-2 下载页面\n",
    "with open (\"weixin_用户研究.html\", encoding = \"utf8\", mode = \"w\") as fp:\n",
    "    fp.write(r.html.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'互联网的兴起使交互设计成为了热门专业,相信很多同学最初接触这个专业时都有很多困惑为什么交互设计师又叫UI/UX设计师?为什...'"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# lxml.text_content()\n",
    "r.html.xpath('//div[contains(@class,\"txt-box\")]/h3/a/em/text()')\n",
    "r.html.xpath('//div[contains(@class,\"txt-box\")]/h3/a/text()')\n",
    "r.html.xpath('//div[contains(@class,\"txt-box\")]/p')[0].lxml.text_content()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdzvlH_a5r65klUuXc2en3Gq_QQkGxcvz01GPlrCXgVybgzbjVvui9cEmt8NBnZRMT3pMkZ85Vs_1kiRUUmb8I5eHjvCaaaY329gr5NzriOvpxZbBnrZzPuJKUiiWOv1GPM&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdzyTcpXFbALUMDXrxlDu4ACaZ-0tHoZsNqxM98YuihUyW1c7TPu0OBx791bLKE-JHMVH-zu5ukyLKYPTNkaddccKymIkEO6BIyHV2HHO0rsZYPy9sdpM6LE6UiiWOv1GPM&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdzLu6p_YHPvqMomX8Jw7s4P98oVSgz3ujg8lI21epsCrqpdk52BWXqQqLvgu3IyWSg5ICIBmXgUSYtTKEfIeb4UiV3jY6jd7Nu1gAxELCSyrMOa501LnEsJWRTpLLTNgHY&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdziNSiL1Avg2ao5rRQX6Hn9SdOk03IphjLaq2-I0yIEo0BfW3Gl-Ne4SHmLj5mnxePnYjhdLG3If-OCgmU-0ZzS1_rPAbBl22Zf_WUciGnMWmUr-ms_JrBqm4OZCnxixtX&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdz2wB3jvhVko_T8ohFL1tGOQh9YzFn6dUMO96SkGcuP9o8DGuuar5-2HRqwvn2cWiqlktbSdI8BGrQDh88nrnM3aVe14Nvsc6lBBg0WAPLmOtmD0-kp0bjg-O00efWrWmm&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdzXMqhETrsdE_RtAPw38mJvPrk99_X7i32HVbaTBmFvVXWokMAVF8SQl22rqYF0afKHpcUOJLPNyGnQg_KWVJ0E6Hqr38LfhMGDdKHKMYwauXSFL9FKDKXlOO00efWrWmm&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdziNSiL1Avg2ao5rRQX6Hn9SdOk03IphjLaq2-I0yIEo0BfW3Gl-Ne4SHmLj5mnxePnYjhdLG3If-OCgmU-0ZzS1_rPAbBl22Zf_WUciGnMWmUr-ms_JrBqm4OZCnxixtX&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdzd8w_aBGyrdDMdhZ06333pbkkKpO8LNnbiEJwA_cW5xILqTSwey0_vK_u_elOboRfIvjX-DV_CXrb6bFIfvVI73pF-ffdx6BuEIZquLMprQDX9HJnGkd9c-O00efWrWmm&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdzZP0DOtsf9_d3n_V_UvYpiGelxaBOYqA2fkAH3nxwaAjPMjpgqhGyWYD1uY3rztj-lydD-1o0_fw9-Tl0CLbolzqiYUQshEIZJz_5X0eKzIajL_AdcVHUlKUiiWOv1GPM&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D',\n",
       " '/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqp8eNnzuFw-CIY2VaTzL3CAwvDqyjOWdziPV4qVS8Ms4dIT8Q-BYpGIbIwSHWLyo54r_EZH0SgAoVx9k_A4MqqL4XPJLw62WrU4s0huyy2TPiE4_Rpa5vk9HIQxCEiZeJgHgpPruSNivYa_HL52XI_eO00efWrWmm&type=2&query=UX&token=398A987010187BEE30359285CC7AB813313233B25E9D623D']"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.html.xpath('//div[contains(@class,\"txt-box\")]/h3/a/@href')\n",
    "r.html.xpath('//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/a/@href')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.request.headers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 下载页面和输出excel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A-2 下载页面\n",
    "with open (\"weixin_python.html\", encoding = \"utf8\", mode = \"w\") as fp:\n",
    "    fp.write(r.html.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A-3 输出excel档\n",
    "数据.to_excel(\"20春_Web数据挖掘_week05_sougou_weixin.xlsx\", sheet_name=\"搜查结果\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 拆分"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 文章链接"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgS8HGqhNZfMkjpkPNaHr35efNqxEqhR43d1qXa8Fplpd9yuT2QY54enf9bAksingjLCs_KuRZJl5NBxx6Eg8BTjDUjKBdcyDEue1iuztX0K70smVcfQciKgIECCZf6SbZd-BUdmK7pbvA0zwZedSM4QPZyYwczruvb4v8obPfPfaX8I_2bc-Eu-qHyS1TmAK_Sw7V0vBReqKvYk0LYiYiS_jf3q8uRDTfkw..&type=2&query=UX&token=39906D0F939BF86DB3B61007A27F1075B374CEB85E9D6395',\n",
       " 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgS8HGqhNZfMkjpkPNaHr35efNqxEqhR43d1qXa8Fplpd9Gd6aW1bwTXnKzJTUwqn686-RqivjhFfhMK6NdilwTkfvNO_kttJ8oT61d-GPgpxJVGDgWi6ml4RbrQJeSQmmFNVcLmuRrOfxc63dmuaEmpqRHBfE3SczB-tyXXE4wEIN2HzC8WobOnmfQXt6OIwOQA0Ag8olTFgLtQZWWeNhCW7P28temKTOwg..&type=2&query=UX&token=39906D0F939BF86DB3B61007A27F1075B374CEB85E9D6395',\n",
       " 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgS8HGqhNZfMkjpkPNaHr35efNqxEqhR43d1qXa8Fplpd9qHyIhFoGHgei_J-YSckMdbpMrnKNENoNlJzWxEPCdbOLpsGnsUOKUacxLJL2Wp0Bd_KGSronijCW0ET1Er7QjE0U1cUhYopDO1yNCPtDYnDQ110Mkr4YxFTbGtjHgIwpAirMPtdrNiIj_R0hNzrv_y6oMDNQ6Us4gU-a7OKwT5cC2Qmj7RgGwQ..&type=2&query=UX&token=39906D0F939BF86DB3B61007A27F1075B374CEB85E9D6395',\n",
       " 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgS8HGqhNZfMkjpkPNaHr35efNqxEqhR43d1qXa8Fplpd9zUHHWWRa-mMUuvWb3MkAPPXtTlCFY8FjrE36n3II8udnsRZkgLKHfxDTFPT6rZuH9XgFCUW8GHaKHR_xl42oadQwBo2ZaS8WRceXAkQvqqFCQY-cG8hxUwOKPGfJ_c49WE6_WO4ss3EP8g3jxbtaPGHmvBOu0pguy7HKBtNvRG3Yl_Q5RRZQjg..&type=2&query=UX&token=39906D0F939BF86DB3B61007A27F1075B374CEB85E9D6395',\n",
       " 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgS8HGqhNZfMkjpkPNaHr35efNqxEqhR43d1qXa8Fplpd9zDkO8R9kSSw9_92jCQtxc_iiq600Bubn_TehRD8fFdinY-OaoFEvRUp26pASMp7GlgFT3O14huDt5olAwoRZA4GNw73QpqCSFK7lA8KroUy0rkQGsuCB1tXpHddaDJ8fQr7dSonJVX99MEYr9SoLEnuBasJQuNn9BoP9VimiBOvuL0unUdKkuw..&type=2&query=UX&token=39906D0F939BF86DB3B61007A27F1075B374CEB85E9D6395',\n",
       " 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgS8HGqhNZfMkjpkPNaHr35efNqxEqhR43d1qXa8Fplpd9tgIvW-dCAET1KgQpl3m_G2yD9-fFEzMNSBqmoDhwBVQVJkvTGBO8mHQCCszoiFhxUIw1wd5vJAaB_2Eyda2u59JGd20PHsoBut7LimpFteNsuY3ARRuTa0qUaYVyJ-5TNKG0FquBjUXpQUJpZ4iBmfR2utwN178A0EwbD8T8FoJflztgI7RZ1g..&type=2&query=UX&token=39906D0F939BF86DB3B61007A27F1075B374CEB85E9D6395',\n",
       " 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgS8HGqhNZfMkjpkPNaHr35efNqxEqhR43d1qXa8Fplpd9zUHHWWRa-mMUuvWb3MkAPPXtTlCFY8FjrE36n3II8udnsRZkgLKHf80ONu97Y82NlwX2XRQxhhmuImrAvbtuUTodIcREuDQ3tveexcK-ml2PT10YLnNBwVGwIj7EMbvTby512Ks74toVkUwmAjca7KOerUu2M3SAAo7xbtuPiCPS-e4Yz84xMA..&type=2&query=UX&token=39906D0F939BF86DB3B61007A27F1075B374CEB85E9D6395',\n",
       " 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgS8HGqhNZfMkjpkPNaHr35efNqxEqhR43d1qXa8Fplpd9yYixHZmrDxEKbwnORtylefWCfUfLK4H39FZJwDwRElxXo5E-287MnsYpsh0t6XlVBrPcOUW0nseqSTXsBP_3VYaVNcanIl6Rm9lzsNgEXKlx7g19P_ZMESsTM8NHPB2iOFQMavtZbY2YNzmLz28cOv_3sEQSS3tCbjBH6MFmKzlFkDr8IT4KYw..&type=2&query=UX&token=39906D0F939BF86DB3B61007A27F1075B374CEB85E9D6395',\n",
       " 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgS8HGqhNZfMkjpkPNaHr35efNqxEqhR43d1qXa8Fplpd94imBcnNG5OArVPcegJqk08ooi6jyORGFSRb29qnQsYE0v9JE10XQCmvHt640Zi_97ZnMa0cijcltMGEcidd6abp5qx9O83MNyuUFQiL1dDMNh5VwT3cfPnCJKbRqaA6Qgl45fNc8kWHLPi0YFmrMYkY3VUEJhJrC0VrpvWmPiX_0OK9DLfgmPA..&type=2&query=UX&token=39906D0F939BF86DB3B61007A27F1075B374CEB85E9D6395',\n",
       " 'https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgS8HGqhNZfMkjpkPNaHr35efNqxEqhR43d1qXa8Fplpd9IsGN4_lqFnr-ZcAnPpM1Rl-sS1lBpsugM1NLY_RAcK_LhIMg3Vyi3RcuwIEWORNLMKx80gMEqb4gAgbbmUHXG-WykQW5Y2VSqLwToMSqGFy6pdJeRNhiGWQKIqh9krNSvRkGNgaqBb9lRyxzyqgNaDjUYiPa91h6hmOPUV6cyIbm0CtGSSkZQQ..&type=2&query=UX&token=39906D0F939BF86DB3B61007A27F1075B374CEB85E9D6395']"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A-4 文章链接 细化\n",
    "# href_list 文章链接\n",
    "article_href_list = []\n",
    "for item in r.html.xpath('//div[contains(@class,\"txt-box\")]/h3/a/@href'):\n",
    "#     print(item)\n",
    "    article_href_list.append(\"https://weixin.sogou.com\"+item)\n",
    "article_href_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 公众号链接"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqrCFjpf57Qst5Icl_Q60XNgwvDqyjOWdzvlH_a5r65klUuXc2en3Gq_QQkGxcvz01GPlrCXgVybgzbjVvui9cEmt8NBnZRMT3pMkZ85Vs_1kiRUUmb8I5eLyqRIjGIRLGSVBBoZ7rDf2c6Wv8bKAEUm4OZCnxixtX&type=2&query=UX&token=39906D0F939BF86DB3B61007A27F1075B374CEB85E9D6395',\n",
       " 'http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqrCFjpf57Qst5Icl_Q60XNgwvDqyjOWdzyTcpXFbALUMDXrxlDu4ACaZ-0tHoZsNqxM98YuihUyW1c7TPu0OBx791bLKE-JHMVH-zu5ukyLKYPTNkaddccFpfxpH_7y9Qyr0-HmXmR5AjNjNxIBDso-O00efWrWmm&type=2&query=UX&token=39906D0F939BF86DB3B61007A27F1075B374CEB85E9D6395',\n",
       " 'http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqrCFjpf57Qst5Icl_Q60XNgwvDqyjOWdzLu6p_YHPvqMomX8Jw7s4P98oVSgz3ujg8lI21epsCrqpdk52BWXqQqLvgu3IyWSg5ICIBmXgUSYtTKEfIeb4UoP2PapxmcDkjpRKI-HRf2fWNWY08bLG5eO00efWrWmm&type=2&query=UX&token=39906D0F939BF86DB3B61007A27F1075B374CEB85E9D6395',\n",
       " 'http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqrCFjpf57Qst5Icl_Q60XNgwvDqyjOWdziNSiL1Avg2ao5rRQX6Hn9SdOk03IphjLaq2-I0yIEo0BfW3Gl-Ne4SHmLj5mnxePnYjhdLG3If-OCgmU-0ZzS3cEiASzU-pXIfSl3qfAB5cJyVo1smLv6G4OZCnxixtX&type=2&query=UX&token=39906D0F939BF86DB3B61007A27F1075B374CEB85E9D6395',\n",
       " 'http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqrCFjpf57Qst5Icl_Q60XNgwvDqyjOWdz2wB3jvhVko_T8ohFL1tGOQh9YzFn6dUMO96SkGcuP9o8DGuuar5-2HRqwvn2cWiqlktbSdI8BGrQDh88nrnM3e2y_31xARlL4yrjyRrlwqi_Pir-A43lzW4OZCnxixtX&type=2&query=UX&token=39906D0F939BF86DB3B61007A27F1075B374CEB85E9D6395',\n",
       " 'http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqrCFjpf57Qst5Icl_Q60XNgwvDqyjOWdzXMqhETrsdE_RtAPw38mJvPrk99_X7i32HVbaTBmFvVXWokMAVF8SQl22rqYF0afKHpcUOJLPNyGnQg_KWVJ0E8T4zTsubvJSQhG2nVdEfVQPNbZ5ZkpxRqUiiWOv1GPM&type=2&query=UX&token=39906D0F939BF86DB3B61007A27F1075B374CEB85E9D6395',\n",
       " 'http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqrCFjpf57Qst5Icl_Q60XNgwvDqyjOWdziNSiL1Avg2ao5rRQX6Hn9SdOk03IphjLaq2-I0yIEo0BfW3Gl-Ne4SHmLj5mnxePnYjhdLG3If-OCgmU-0ZzS3cEiASzU-pXIfSl3qfAB5cJyVo1smLv6G4OZCnxixtX&type=2&query=UX&token=39906D0F939BF86DB3B61007A27F1075B374CEB85E9D6395',\n",
       " 'http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqrCFjpf57Qst5Icl_Q60XNgwvDqyjOWdzd8w_aBGyrdDMdhZ06333pbkkKpO8LNnbiEJwA_cW5xILqTSwey0_vK_u_elOboRfIvjX-DV_CXrb6bFIfvVI726zBsMhUzAmY2iyQl9Akak3WvfmZO4ESOO00efWrWmm&type=2&query=UX&token=39906D0F939BF86DB3B61007A27F1075B374CEB85E9D6395',\n",
       " 'http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqrCFjpf57Qst5Icl_Q60XNgwvDqyjOWdzZP0DOtsf9_d3n_V_UvYpiGelxaBOYqA2fkAH3nxwaAjPMjpgqhGyWYD1uY3rztj-lydD-1o0_fw9-Tl0CLbol6XE5xM63fjBJIsx2rdOspi2IEvg8C1-M-O00efWrWmm&type=2&query=UX&token=39906D0F939BF86DB3B61007A27F1075B374CEB85E9D6395',\n",
       " 'http://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6EzDJysI4ql5MPrOUp16838dGRMI7NnPqrCFjpf57Qst5Icl_Q60XNgwvDqyjOWdziPV4qVS8Ms4dIT8Q-BYpGIbIwSHWLyo54r_EZH0SgAoVx9k_A4MqqL4XPJLw62WrU4s0huyy2TPiE4_Rpa5vk8AaeR2s7yxcDyPo2WZdHEaI-lZPP84r9qUiiWOv1GPM&type=2&query=UX&token=39906D0F939BF86DB3B61007A27F1075B374CEB85E9D6395']"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A-5 公众号链接 细化\n",
    "[\"http://weixin.sogou.com\"+item\\\n",
    " for e in 主要元素 for item in e.xpath('//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/a/@href')]\n",
    " #for e in 主要元素]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 来源和发表时间"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A-6 来源和发表时间 细化\n",
    "source = r.html.xpath('//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/a/text()')\n",
    "time =  r.html.xpath('//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/@t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 翻页：参数字典的拆解\n",
    "\n",
    "## xpath解析翻页a/@href"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# B-0   单一页面\n",
    "url = \"https://weixin.sogou.com/weixin\"\n",
    "session = HTMLSession()\n",
    "payload = {\n",
    "    \"query\":\"python\",\n",
    "    \"type\":\"2\"\n",
    "}\n",
    "\n",
    "r = session.get( url, params = payload)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element 'a' id='sogou_page_2' href='?query=python&type=2&page=2&ie=utf8' uigs='page_2'>, <Element 'a' id='sogou_page_3' href='?query=python&type=2&page=3&ie=utf8' uigs='page_3'>, <Element 'a' id='sogou_page_4' href='?query=python&type=2&page=4&ie=utf8' uigs='page_4'>, <Element 'a' id='sogou_page_5' href='?query=python&type=2&page=5&ie=utf8' uigs='page_5'>, <Element 'a' id='sogou_page_6' href='?query=python&type=2&page=6&ie=utf8' uigs='page_6'>, <Element 'a' id='sogou_page_7' href='?query=python&type=2&page=7&ie=utf8' uigs='page_7'>, <Element 'a' id='sogou_page_8' href='?query=python&type=2&page=8&ie=utf8' uigs='page_8'>, <Element 'a' id='sogou_page_9' href='?query=python&type=2&page=9&ie=utf8' uigs='page_9'>, <Element 'a' id='sogou_page_10' href='?query=python&type=2&page=10&ie=utf8' uigs='page_10'>, <Element 'a' id='sogou_next' href='?query=python&type=2&page=2&ie=utf8' class=('np',) uigs='page_next'>]\n",
      "['?query=python&type=2&page=2&ie=utf8', '?query=python&type=2&page=3&ie=utf8', '?query=python&type=2&page=4&ie=utf8', '?query=python&type=2&page=5&ie=utf8', '?query=python&type=2&page=6&ie=utf8', '?query=python&type=2&page=7&ie=utf8', '?query=python&type=2&page=8&ie=utf8', '?query=python&type=2&page=9&ie=utf8', '?query=python&type=2&page=10&ie=utf8', '?query=python&type=2&page=2&ie=utf8']\n",
      "{'2': '?query=python&type=2&page=2&ie=utf8', '3': '?query=python&type=2&page=3&ie=utf8', '4': '?query=python&type=2&page=4&ie=utf8', '5': '?query=python&type=2&page=5&ie=utf8', '6': '?query=python&type=2&page=6&ie=utf8', '7': '?query=python&type=2&page=7&ie=utf8', '8': '?query=python&type=2&page=8&ie=utf8', '9': '?query=python&type=2&page=9&ie=utf8', '10': '?query=python&type=2&page=10&ie=utf8', '下一页': '?query=python&type=2&page=2&ie=utf8'}\n"
     ]
    }
   ],
   "source": [
    "# A-1  xpath 解析翻页a/@href\n",
    "xpath_翻页a = '//div[@class=\"p-fy\"]/a'\n",
    "xpath_翻页a = '//div[@class=\"p-fy\"]/a'\n",
    "print (r.html.xpath(xpath_翻页a))\n",
    "\n",
    "href_列表 = [x.xpath('//@href')[0] for x in r.html.xpath(xpath_翻页a)]\n",
    "print (href_列表)\n",
    "\n",
    "href_字典 = {x.text:x.xpath('//@href')[0]  for x in r.html.xpath(xpath_翻页a)}\n",
    "print (href_字典)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'query': ['python'], 'type': ['2'], 'page': ['2'], 'ie': ['utf8']}"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parse_qs(\"query=python&type=2&page=2&ie=utf8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scheme</th>\n",
       "      <th>netloc</th>\n",
       "      <th>path</th>\n",
       "      <th>params</th>\n",
       "      <th>query</th>\n",
       "      <th>fragment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>query=python&amp;type=2&amp;page=2&amp;ie=utf8</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>query=python&amp;type=2&amp;page=3&amp;ie=utf8</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>query=python&amp;type=2&amp;page=4&amp;ie=utf8</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>query=python&amp;type=2&amp;page=5&amp;ie=utf8</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>query=python&amp;type=2&amp;page=6&amp;ie=utf8</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>query=python&amp;type=2&amp;page=7&amp;ie=utf8</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>query=python&amp;type=2&amp;page=8&amp;ie=utf8</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>query=python&amp;type=2&amp;page=9&amp;ie=utf8</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>query=python&amp;type=2&amp;page=10&amp;ie=utf8</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>query=python&amp;type=2&amp;page=2&amp;ie=utf8</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  scheme netloc path params                                query fragment\n",
       "0                             query=python&type=2&page=2&ie=utf8         \n",
       "1                             query=python&type=2&page=3&ie=utf8         \n",
       "2                             query=python&type=2&page=4&ie=utf8         \n",
       "3                             query=python&type=2&page=5&ie=utf8         \n",
       "4                             query=python&type=2&page=6&ie=utf8         \n",
       "5                             query=python&type=2&page=7&ie=utf8         \n",
       "6                             query=python&type=2&page=8&ie=utf8         \n",
       "7                             query=python&type=2&page=9&ie=utf8         \n",
       "8                            query=python&type=2&page=10&ie=utf8         \n",
       "9                             query=python&type=2&page=2&ie=utf8         "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scheme      1\n",
      "netloc      1\n",
      "path        1\n",
      "params      1\n",
      "query       9\n",
      "fragment    1\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query</th>\n",
       "      <th>type</th>\n",
       "      <th>page</th>\n",
       "      <th>ie</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>utf8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>utf8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>utf8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>utf8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>utf8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>utf8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>utf8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>utf8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>utf8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>utf8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    query type page    ie\n",
       "0  python    2    2  utf8\n",
       "1  python    2    3  utf8\n",
       "2  python    2    4  utf8\n",
       "3  python    2    5  utf8\n",
       "4  python    2    6  utf8\n",
       "5  python    2    7  utf8\n",
       "6  python    2    8  utf8\n",
       "7  python    2    9  utf8\n",
       "8  python    2   10  utf8\n",
       "9  python    2    2  utf8"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "query    1\n",
      "type     1\n",
      "page     9\n",
      "ie       1\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query</th>\n",
       "      <th>type</th>\n",
       "      <th>page</th>\n",
       "      <th>ie</th>\n",
       "      <th>page_int</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>utf8</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>utf8</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>utf8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>utf8</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>utf8</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>utf8</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>utf8</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>utf8</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>utf8</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>python</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>utf8</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    query type page    ie  page_int\n",
       "0  python    2    2  utf8         2\n",
       "1  python    2    3  utf8         3\n",
       "2  python    2    4  utf8         4\n",
       "3  python    2    5  utf8         5\n",
       "4  python    2    6  utf8         6\n",
       "5  python    2    7  utf8         7\n",
       "6  python    2    8  utf8         8\n",
       "7  python    2    9  utf8         9\n",
       "8  python    2   10  utf8        10\n",
       "9  python    2    2  utf8         2"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# A-2 建构参数模板：找到关键参数及参数结构\n",
    "\n",
    "# 需要模组库\n",
    "from urllib.parse import urlparse, parse_qs\n",
    "import pandas as pd\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "\n",
    "\n",
    "df = pd.DataFrame([ urlparse(x) for x in href_列表])\n",
    "\n",
    "df_qs = pd.DataFrame([{k:v[0] for k,v in parse_qs(x).items()} for x in df['query'] ])\n",
    "\n",
    "display(df)\n",
    "\n",
    "\n",
    "\n",
    "print(df.nunique())\n",
    "display(df_qs)\n",
    "print(df_qs.nunique())\n",
    "\n",
    "df_qs.page\n",
    "df_qs = df_qs.assign (page_int=df_qs.page.astype(int)) # 变成整数(数值)\n",
    "display(df_qs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'query': ['python'], 'type': ['2'], 'page': ['4'], 'ie': ['utf8']}\n",
      "{'2': '?query=python&type=2&page=2&ie=utf8', '3': '?query=python&type=2&page=3&ie=utf8', '4': '?query=python&type=2&page=4&ie=utf8', '5': '?query=python&type=2&page=5&ie=utf8', '6': '?query=python&type=2&page=6&ie=utf8', '7': '?query=python&type=2&page=7&ie=utf8', '8': '?query=python&type=2&page=8&ie=utf8', '9': '?query=python&type=2&page=9&ie=utf8', '10': '?query=python&type=2&page=10&ie=utf8', '下一页': '?query=python&type=2&page=2&ie=utf8'}\n"
     ]
    }
   ],
   "source": [
    "# A-2 建构参数模板：找到关键参数及参数结构\n",
    "\n",
    "def parse_url_qs_for_page (url):\n",
    "    six_parts = urlparse(url) \n",
    "    out = parse_qs(six_parts.query)\n",
    "    return (out)\n",
    "\n",
    "# 取一例做模板\n",
    "参数模板 = parse_url_qs_for_page(href_列表[2])\n",
    "print (参数模板)\n",
    "\n",
    "# 重新审视：href_字典\n",
    "print (href_字典)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "10\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{1: {'query': ['用户体验'], 'type': ['2'], 'page': [1], 'ie': ['utf8']},\n",
       " 2: {'query': ['用户体验'], 'type': ['2'], 'page': [2], 'ie': ['utf8']},\n",
       " 3: {'query': ['用户体验'], 'type': ['2'], 'page': [3], 'ie': ['utf8']},\n",
       " 4: {'query': ['用户体验'], 'type': ['2'], 'page': [4], 'ie': ['utf8']},\n",
       " 5: {'query': ['用户体验'], 'type': ['2'], 'page': [5], 'ie': ['utf8']},\n",
       " 6: {'query': ['用户体验'], 'type': ['2'], 'page': [6], 'ie': ['utf8']},\n",
       " 7: {'query': ['用户体验'], 'type': ['2'], 'page': [7], 'ie': ['utf8']},\n",
       " 8: {'query': ['用户体验'], 'type': ['2'], 'page': [8], 'ie': ['utf8']},\n",
       " 9: {'query': ['用户体验'], 'type': ['2'], 'page': [9], 'ie': ['utf8']},\n",
       " 10: {'query': ['用户体验'], 'type': ['2'], 'page': [10], 'ie': ['utf8']}}"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A-3 建构参数模板生成器：query（搜索关键字）page\n",
    "def 参数模板生成(query, page):\n",
    "    参数 = 参数模板.copy()\n",
    "    参数['page'] = page\n",
    "    参数['query'] = query\n",
    "    return (参数)\n",
    "\n",
    "# -----------    参数模版生成器 目的 构建payload\n",
    "# payload={\n",
    "    \n",
    "# }\n",
    "参数_query_用户体验_page = { \n",
    "    i:参数模板生成(page = [i], \\\n",
    "                  query = ['用户体验']) \\\n",
    "    for i,v in href_字典.items()\\\n",
    "    }\n",
    "\n",
    "print (df_qs.page_int.min()) # 最小值只有2 1的时候没有链接？\n",
    "print (df_qs.page_int.max()) # 最大值只有9\n",
    "\n",
    "参数_query_用户体验_page = { \n",
    "    i:参数模板生成(page = [i], \\\n",
    "                  query = ['用户体验']) \\\n",
    "    for i in range(df_qs.page_int.min()-1,df_qs.page_int.max()+1)\\\n",
    "    }\n",
    "参数_query_用户体验_page"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 封装request_sougou_weixin：（练习：修改参数和页数）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 1 页\n",
      "payload: {'query': '如果历史是一群猫', 'type': '2', 'page': 1}\n",
      "url: https://weixin.sogou.com/weixin\n",
      "第 1 页 的Dataframe\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>《如果历史是一群喵》:这是什么喵星福利,云吸猫的同时还能变学霸</td>\n",
       "      <td>这部动画就是《如果历史是一群喵》.作品简介《如果历史是一群喵》是一部以华夏历史为主线,依据二...</td>\n",
       "      <td>国漫新视点</td>\n",
       "      <td>1585917558</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-03 12:39:18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>推荐 | 《如果历史是一群喵》</td>\n",
       "      <td>今天推荐的历史动画是《如果历史是一群喵》.作品画风、文风都非常幽默,以诙谐幽默又接地气的方式...</td>\n",
       "      <td>福禄贝尔国际儿童之家</td>\n",
       "      <td>1586578116</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-11 04:08:36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>“捧曹操,踩刘备”,《如果历史是一群喵》该不该停更?</td>\n",
       "      <td>题图 / 如果历史是一群喵本文由ACGx原创,转载请注明出处.停更,并不能解决所有问题3月3...</td>\n",
       "      <td>ACGx</td>\n",
       "      <td>1586508246</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-10 08:44:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>爆火历史动画|《如果历史是一群喵》</td>\n",
       "      <td>《如果历史是一群喵》之《华夏诞生》《如果历史是一群喵》用幽默风趣的语言对历史事件进行了重新解...</td>\n",
       "      <td>车库创客机器人中心</td>\n",
       "      <td>1586587574</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-11 06:46:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>与孩子的共读时光 ▏《如果历史是一群喵》萌猫助你历史得满分!</td>\n",
       "      <td>《如果历史是一群喵》用幽默风趣的喵民们来讲述中华上下5000年的兴衰迭起.严格意义上来讲这本...</td>\n",
       "      <td>大连城市书房</td>\n",
       "      <td>1574236860</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2019-11-20 08:01:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>免费分享 |  大开眼界!如果历史是一群喵...</td>\n",
       "      <td>如果你家孩子也不喜欢历史,一定一定要跟你推荐这部历史启蒙动画——《如果历史是一群喵》.就如同...</td>\n",
       "      <td>怡敦英语</td>\n",
       "      <td>1587219848</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-18 14:24:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>爆火是历史动画《如果历史是一群喵》1-30集</td>\n",
       "      <td>好看的都关注了我们爆火是历史动画《如果历史是一群喵》《如果历史是一群喵》第1-3 回爆火历史...</td>\n",
       "      <td>初高中历史微课堂</td>\n",
       "      <td>1587254450</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-19 00:00:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>《如果历史是一群喵》(23-25集)</td>\n",
       "      <td>本视频来自网络,让学生学习之余放松之用,如有侵权联系删《如果历史是一群喵》第23回  昭王称...</td>\n",
       "      <td>历史微课</td>\n",
       "      <td>1582819258</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-02-27 16:00:58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>爆火历史动画《如果历史是一群喵》第26回 千秋一统</td>\n",
       "      <td>《如果历史是一群喵》是一部以华夏历史为主线,依据二十四史等文献资料编绘的历史漫画,旨在向青年...</td>\n",
       "      <td>初高中历史微课堂</td>\n",
       "      <td>1585789259</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2020-04-02 01:00:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>每日一书|《如果历史是一群猫》(3册)</td>\n",
       "      <td>每日一书|今日推荐书名:如果历史是一群猫(全书三册)作者:肥志出版日期:2019-06定价:...</td>\n",
       "      <td>单县新华书店信息发布平台</td>\n",
       "      <td>1573703414</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>https://weixin.sogou.com/link?url=dn9a_-gY295K...</td>\n",
       "      <td>2019-11-14 03:50:14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                标题  \\\n",
       "0  《如果历史是一群喵》:这是什么喵星福利,云吸猫的同时还能变学霸   \n",
       "1                  推荐 | 《如果历史是一群喵》   \n",
       "2       “捧曹操,踩刘备”,《如果历史是一群喵》该不该停更?   \n",
       "3                爆火历史动画|《如果历史是一群喵》   \n",
       "4   与孩子的共读时光 ▏《如果历史是一群喵》萌猫助你历史得满分!   \n",
       "5         免费分享 |  大开眼界!如果历史是一群喵...   \n",
       "6           爆火是历史动画《如果历史是一群喵》1-30集   \n",
       "7               《如果历史是一群喵》(23-25集)   \n",
       "8        爆火历史动画《如果历史是一群喵》第26回 千秋一统   \n",
       "9              每日一书|《如果历史是一群猫》(3册)   \n",
       "\n",
       "                                                  摘要        source  \\\n",
       "0  这部动画就是《如果历史是一群喵》.作品简介《如果历史是一群喵》是一部以华夏历史为主线,依据二...         国漫新视点   \n",
       "1  今天推荐的历史动画是《如果历史是一群喵》.作品画风、文风都非常幽默,以诙谐幽默又接地气的方式...    福禄贝尔国际儿童之家   \n",
       "2  题图 / 如果历史是一群喵本文由ACGx原创,转载请注明出处.停更,并不能解决所有问题3月3...          ACGx   \n",
       "3  《如果历史是一群喵》之《华夏诞生》《如果历史是一群喵》用幽默风趣的语言对历史事件进行了重新解...     车库创客机器人中心   \n",
       "4  《如果历史是一群喵》用幽默风趣的喵民们来讲述中华上下5000年的兴衰迭起.严格意义上来讲这本...        大连城市书房   \n",
       "5  如果你家孩子也不喜欢历史,一定一定要跟你推荐这部历史启蒙动画——《如果历史是一群喵》.就如同...          怡敦英语   \n",
       "6  好看的都关注了我们爆火是历史动画《如果历史是一群喵》《如果历史是一群喵》第1-3 回爆火历史...      初高中历史微课堂   \n",
       "7  本视频来自网络,让学生学习之余放松之用,如有侵权联系删《如果历史是一群喵》第23回  昭王称...          历史微课   \n",
       "8  《如果历史是一群喵》是一部以华夏历史为主线,依据二十四史等文献资料编绘的历史漫画,旨在向青年...      初高中历史微课堂   \n",
       "9  每日一书|今日推荐书名:如果历史是一群猫(全书三册)作者:肥志出版日期:2019-06定价:...  单县新华书店信息发布平台   \n",
       "\n",
       "           时间                                              公众号链接  \\\n",
       "0  1585917558  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "1  1586578116  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "2  1586508246  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "3  1586587574  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "4  1574236860  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "5  1587219848  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "6  1587254450  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "7  1582819258  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "8  1585789259  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "9  1573703414  https://weixin.sogou.com/link?url=dn9a_-gY295K...   \n",
       "\n",
       "                                                文章链接                格式化时间  \n",
       "0  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-03 12:39:18  \n",
       "1  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-11 04:08:36  \n",
       "2  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-10 08:44:06  \n",
       "3  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-11 06:46:14  \n",
       "4  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2019-11-20 08:01:00  \n",
       "5  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-18 14:24:08  \n",
       "6  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-19 00:00:50  \n",
       "7  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-02-27 16:00:58  \n",
       "8  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2020-04-02 01:00:59  \n",
       "9  https://weixin.sogou.com/link?url=dn9a_-gY295K...  2019-11-14 03:50:14  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "session = HTMLSession()\n",
    "def request_sougou_weixin(url,params):\n",
    "   \n",
    "    r = session.get( url, params = payload, )\n",
    "    \n",
    "    \n",
    "    主要元素 =  r.html.xpath( '//ul[@class=\"news-list\"]/li')\n",
    "\n",
    "    dict_xpaths = {\n",
    "        'text':{\n",
    "            'source':'//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/a/text()',\n",
    "            '时间':'//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/@t',       \n",
    "        },\n",
    "        'text_content':{\n",
    "            '标题':'//div[contains(@class,\"txt-box\")]/h3/a',\n",
    "            '摘要':'//div[contains(@class,\"txt-box\")]/p'\n",
    "        },\n",
    "        'href':{\n",
    "            \"公众号链接\":'//div[contains(@class,\"txt-box\")]/div[@class=\"s-p\"]/a/@href',\n",
    "            \"文章链接\":'//div[contains(@class,\"txt-box\")]/h3/a/@href',\n",
    "        }\n",
    "    }\n",
    "\n",
    "\n",
    "    def get_e_text_content(_xpath_):\n",
    "        # 高级列表推导\n",
    "        暂存结果 = [e.xpath(_xpath_)[0].lxml.text_content() for e in 主要元素]\n",
    "\n",
    "        return(暂存结果)\n",
    "\n",
    "    def get_e_text(_xpath_):\n",
    "        暂存结果 = [\"\".join([x.strip() if type(x) is str else x.text.strip() for x in e.xpath(_xpath_)]) for e in 主要元素]\n",
    "        return(暂存结果)\n",
    "\n",
    "    def get_e_href(_xpath_):\n",
    "        # 高级列表推导\n",
    "        暂存结果 = [\"https://weixin.sogou.com\"+item\\\n",
    "                for e in 主要元素\\\n",
    "                for item in e.xpath(_xpath_)\\\n",
    "                ] #此处依旧报错，修改了顺序\n",
    "        return(暂存结果)\n",
    "\n",
    "    数据字典 = dict()\n",
    "    数据字典 = {k:get_e_text_content(v) for k,v in dict_xpaths['text_content'].items()}\n",
    "\n",
    "    数据字典.update({k:get_e_text(v) for k,v in dict_xpaths['text'].items()})\n",
    "    数据字典.update({k:get_e_href(v) for k,v in dict_xpaths['href'].items()})\n",
    "\n",
    "    数据 = pd.DataFrame(数据字典)\n",
    "    数据 = 数据.assign(\n",
    "        格式化时间 = lambda x: pd.to_datetime(x[\"时间\"], unit='s').dt.strftime('%Y-%m-%d %H:%M:%S')\n",
    "    )\n",
    "    return(数据)\n",
    "\n",
    "\n",
    "url = \"https://weixin.sogou.com/weixin\"\n",
    "\n",
    "\n",
    "list_df=[]\n",
    "for i in range(1,2):\n",
    "    print(\"第\",i,\"页\")\n",
    "    payload = {\n",
    "        \"query\":\"如果历史是一群猫\",\n",
    "        \"type\":\"2\",\n",
    "        \"page\":i\n",
    "\n",
    "    } \n",
    "    print(\"payload:\",payload)\n",
    "    print(\"url:\",url)\n",
    "    time.sleep(10+10*random())\n",
    "    df = request_sougou_weixin(url,params=payload)\n",
    "    print(\"第\",i,\"页\",\"的Dataframe\")\n",
    "    display(df)\n",
    "    \n",
    "    df.to_csv(\"20春_Web数据挖掘_week05_sougou_weixin_{key}_{k}.tsv\"\\\n",
    "                  .format(key=\"python\", k=i), sep=\"\\t\", encoding=\"utf8\")\n",
    "        \n",
    "    df = df.assign (query = key)  # 区分  query   \n",
    "    df = df.assign (page = i)  # 区分  page    \n",
    "    list_df.append(df)\n",
    "    \n",
    "\n",
    "df_all = pd.concat(list_df).reset_index()\n",
    "df_all.index.name = '序'\n",
    "\n",
    "df_all.to_excel(\"20春_Web数据挖掘_week05_sougou_weixin_翻页_5.xlsx\",\\\n",
    "                sheet_name=\"_\".join(query))\n",
    "# 预估时间: 4*5秒*10 =200\n",
    "# 预估数量: 4*40*10 =1600"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<PreparedRequest [GET]>"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.request"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 多个页面爬取 ：修改查询内容"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'query': ['如果历史是一群猫'], 'type': ['2'], 'page': ['1'], 'ie': ['utf8']}\n",
      "  scheme netloc path params                                query fragment\n",
      "0                             query=python&type=2&page=2&ie=utf8         \n",
      "1                             query=python&type=2&page=3&ie=utf8         \n",
      "2                             query=python&type=2&page=4&ie=utf8         \n",
      "3                             query=python&type=2&page=5&ie=utf8         \n",
      "4                             query=python&type=2&page=6&ie=utf8         \n",
      "5                             query=python&type=2&page=7&ie=utf8         \n",
      "6                             query=python&type=2&page=8&ie=utf8         \n",
      "7                             query=python&type=2&page=9&ie=utf8         \n",
      "8                            query=python&type=2&page=10&ie=utf8         \n",
      "9                             query=python&type=2&page=2&ie=utf8         \n",
      "如果历史是一群猫 10\n",
      "参数_query_X_page {1: {'query': ['如果历史是一群猫'], 'type': ['2'], 'page': [1], 'ie': ['utf8']}, 2: {'query': ['如果历史是一群猫'], 'type': ['2'], 'page': [2], 'ie': ['utf8']}, 3: {'query': ['如果历史是一群猫'], 'type': ['2'], 'page': [3], 'ie': ['utf8']}, 4: {'query': ['如果历史是一群猫'], 'type': ['2'], 'page': [4], 'ie': ['utf8']}, 5: {'query': ['如果历史是一群猫'], 'type': ['2'], 'page': [5], 'ie': ['utf8']}, 6: {'query': ['如果历史是一群猫'], 'type': ['2'], 'page': [6], 'ie': ['utf8']}, 7: {'query': ['如果历史是一群猫'], 'type': ['2'], 'page': [7], 'ie': ['utf8']}, 8: {'query': ['如果历史是一群猫'], 'type': ['2'], 'page': [8], 'ie': ['utf8']}, 9: {'query': ['如果历史是一群猫'], 'type': ['2'], 'page': [9], 'ie': ['utf8']}}\n",
      "payload {'query': ['如果历史是一群猫'], 'type': ['2'], 'page': [1], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['如果历史是一群猫'], 'type': ['2'], 'page': [2], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['如果历史是一群猫'], 'type': ['2'], 'page': [3], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['如果历史是一群猫'], 'type': ['2'], 'page': [4], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['如果历史是一群猫'], 'type': ['2'], 'page': [5], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['如果历史是一群猫'], 'type': ['2'], 'page': [6], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['如果历史是一群猫'], 'type': ['2'], 'page': [7], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['如果历史是一群猫'], 'type': ['2'], 'page': [8], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "payload {'query': ['如果历史是一群猫'], 'type': ['2'], 'page': [9], 'ie': ['utf8']}\n",
      "https://weixin.sogou.com/weixin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>摘要</th>\n",
       "      <th>source</th>\n",
       "      <th>时间</th>\n",
       "      <th>公众号链接</th>\n",
       "      <th>文章链接</th>\n",
       "      <th>格式化时间</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [标题, 摘要, source, 时间, 公众号链接, 文章链接, 格式化时间]\n",
       "Index: []"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 48.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# C-1 多个页面+多个关键词\n",
    "import time\n",
    "from random import random\n",
    "\n",
    "url = \"https://weixin.sogou.com/weixin\"\n",
    "xpath_翻页a = '//div[@class=\"p-fy\"]/a[starts-with(@href,\"?query\")]'\n",
    "# 多界面（不同的内容（分类、关键词））多页面（翻页）\n",
    "query = ['如果历史是一群猫']\n",
    "list_df = list()\n",
    "\n",
    "## 第一页试探有多长的页面\n",
    "for key in query:\n",
    "    payload = 参数模板生成(query=[key], page=['1'])\n",
    "    print(payload)\n",
    "    df = request_sougou_weixin( url, params = payload)\n",
    "    href_列表 = [x.xpath('//@href')[0] for x in r.html.xpath(xpath_翻页a)]\n",
    "    df = pd.DataFrame([ urlparse(x) for x in href_列表])\n",
    "    print(df)\n",
    "    df_qs = pd.DataFrame([{k:v[0] for k,v in parse_qs(x).items()} for x in df['query']])\n",
    "    df_qs = df_qs.assign (page_int=df_qs.page.astype(int)) # 变成整数\n",
    "    长度 = df_qs.page_int.max()\n",
    "    参数_query_X_page = { \n",
    "        i:参数模板生成(page = [i], \\\n",
    "                      query = [key]) \\\n",
    "        for i in range(df_qs.page_int.min()-1,长度)\\\n",
    "        }\n",
    "    #print (参数_query_X_page)\n",
    "    print (key,长度)\n",
    "    print(\"参数_query_X_page\",参数_query_X_page)\n",
    "    \n",
    "    for k,v in 参数_query_X_page.items():\n",
    "        payload = v\n",
    "        print(\"payload\",payload)\n",
    "        print(url)\n",
    "        df = request_sougou_weixin( url, params = payload)\n",
    "        \n",
    "        time.sleep(3+4*random())  #放慢脚步 3-7秒, 平均约5秒\n",
    "        # 备份\n",
    "        display(df)\n",
    "        df.to_csv(\"20春_Web数据挖掘_week05_sougou_weixin_{key}_{k}.tsv\"\\\n",
    "                  .format(key=key, k=k), sep=\"\\t\", encoding=\"utf8\")\n",
    "        \n",
    "        df = df.assign (query = key)  # 区分  keyword    \n",
    "        df = df.assign (page = k)  # 区分  page    \n",
    "        list_df.append(df)\n",
    "        \n",
    "df_all = pd.concat(list_df).reset_index()\n",
    "df_all.index.name = '序'\n",
    "\n",
    "df_all.to_excel(\"20春_Web数据挖掘_week05_sougou_weixin_翻页_5.xlsx\",\\\n",
    "                sheet_name=\"_\".join(query))\n",
    "# 预估时间: 4*5秒*10 =200\n",
    "# 预估数量: 4*40*10 =1600"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "297.391px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
